﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Threading;
using System.Text.RegularExpressions;

namespace CNSegmenter
{
    class Program
    {
        static void TokenizeFromFile()
        {
            var parts = File.ReadAllText(@".\Model\testFile.txt", Encoding.Default).Split("。：、？“”！，.《》 %@#$^&*()-_+=|\\".ToCharArray());
            List<string> tokenizedWords = new List<string>();
            foreach (var p in parts)
            {
                var tokens = CNSegmenter.Instance().Tokenize(p);
                if (tokens != null)
                {
                    foreach (var word in tokens)
                    {
                        tokenizedWords.Add(word);
                    }
                }
            }
            File.WriteAllLines(@".\Model\tokenizedResult.txt", tokenizedWords.ToArray(), Encoding.Default);
        }

        static private void FindNewWordOnSingleChar()
        {
            var tokens = File.ReadAllLines(@".\Model\tokenizedResult.txt", Encoding.Default).ToList();
            string singleWordText = tokens.Where(token => token.Length == 1).Aggregate((i, j) => "" + i + j);
            Console.WriteLine("length: " + singleWordText.Length.ToString());
            Console.WriteLine("% = " + (tokens.Where(token => token.Length == 1).Count() * 1.0 / tokens.Count).ToString());

            Dictionary<string, int> table = new Dictionary<string, int>();
            int startIndex = 0;
            while (startIndex + 4 <= singleWordText.Length)
            {
                for (int length = 2; length <= 4; length++)
                {
                    string w = singleWordText.Substring(startIndex, length);
                    w = w.Trim(' ', '\0', '\n');
                    if (table.ContainsKey(w))
                    {
                        table[w]++;
                    }
                    else
                    {
                        table.Add(w, 1);
                    }
                }
                startIndex += 4;
            }

            int newWordsCount = 0;
            using (StreamWriter sw = new StreamWriter(@".\Model\oneWord.txt"))
            {
                foreach (var item in table)
                {
                    if (item.Value > 1)
                    {
                        Console.WriteLine(item.Key + "<" + item.Value.ToString() + ">");
                        sw.WriteLine(item.Key + "<" + item.Value.ToString() + ">");
                        newWordsCount++;
                    }
                }
            }
            Console.WriteLine(newWordsCount.ToString() + " new words are found.");
        }

        static private void SpiderRunXinli()
        {
            int numberOfProcesses = 4;
            ManualResetEvent[] manualEvents = new ManualResetEvent[numberOfProcesses];
            int start = 3424;
            for (int i = 0; i < numberOfProcesses; i++)
            {
                Spider spider = new Spider();
                spider.start = start;
                spider.end = start + 100;
                start += 101;
                manualEvents[i] = new ManualResetEvent(false);
                spider.manualEvent = manualEvents[i];
                //spider.RunSpider();
                //spider.manualEvent.Set();
                ThreadPool.QueueUserWorkItem((async =>
                {
                    spider.RunSpider();
                }), null);
            }

            // Since ThreadPool threads are background threads, 
            // wait for the work items to signal before exiting.
            WaitHandle.WaitAll(manualEvents);
            Console.WriteLine("Files written - main exiting.");
        }

        static private void MergeFilsToTotal(string directoryPath, string totalFileName)
        {
            // Regex regex = new Regex("(?<cn>([^A-Za-z0-9,.<>/?!`@#$%^&*()-_+=|\\{};:'\"]|\\s)+)");
            Regex regex = new Regex("<p>.?(?<cn>[^<>]+).?</p>");

            foreach (var filename in Directory.GetFiles(directoryPath, "*.txt", SearchOption.TopDirectoryOnly))
            {
                string rawData = File.ReadAllText(filename, Encoding.Default);
                var result = regex.Matches(rawData);
                string content = "";
                foreach (Match match in result)
                {
                    content += match.Groups["cn"].Value + "\n";
                }
                File.AppendAllText(totalFileName, content, Encoding.Default);
                Console.WriteLine("Finish " + filename);
            }
        }

        static void Main(string[] args)
        {
            // TokenizeFromFile();
            
            // MergeFilsToTotal(@".\Data\1", @".\Data\total.txt");

            CNSegmenter.Instance().LearnFromTrainingFile();
            TokenizeFromFile();
            
            
        }
    }
}
